package utils.httpclient;

import cn.hutool.core.io.FileUtil;
import main.pmc.LocalPmcFileCheck2;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.*;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.util.EntityUtils;
import utils.SmallTool;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;

public class HttpClient {


    /**
     * 转化Caoliu返回的html
     */
    public static String parsingHtmlCaoliu(CloseableHttpResponse responce) throws Exception {
        String html = null;
        HttpEntity entity = responce.getEntity();
        if (entity != null) {
            html = EntityUtils.toString(entity, "UTF-8");
            html = html.replace("&nbsp;", " ");
        }
        return html;
    }

    /**
     * 通用的获取Caoliu html的get方法
     */
    public static String getHtmlCaoliu(BasicCookieStore cookieStore, HttpClientConfig config) throws IOException {
//        SmallTool.printMess("当前的地址:" + config.getUrl());
        String html = "";
        for (int i = 1; i <= 3; i++) {
            CloseableHttpClient httpClient;
            if (config.isProxy()) {
                HttpHost httpHost = new HttpHost(config.getProxyIp(), config.getProxyPort());
                httpClient = HttpClientBuilder.create()
                        .setProxy(httpHost)
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();
            } else {
                httpClient = HttpClientBuilder.create()
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();
            }
            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                httpget.addHeader(HttpHeaders.USER_AGENT, config.getUSER_AGENT());
                //添加cookie 测试
//                httpget.addHeader("Cookie","cookiePolicy=iaccept; __gads=ID=241348f041e0579a:T=1547694309:S=ALNI_MYKZ_CVaSL8AL9PVJond_pN6K1NWQ; QSI_SI_29r6u67C30RWd01_intercept=true; AMCVS_1B6E34B85282A0AC0A490D44%40AdobeOrg=1; randomizeUser=0.7046080839406554; cookiePolicy=iaccept; s_vnum=1550286215643%26vn%3D5; s_invisit=true; s_visit=1; SERVER=WZ6myaEXBLHrdNmd/vcq8Q==; SERVER=WZ6myaEXBLHrdNmd/vcq8Q==; MAID=+TSVQKGKfjS/hXTPEnQenQ==; MAID=+TSVQKGKfjS/hXTPEnQenQ==; MACHINE_LAST_SEEN=2019-01-20T01%3A45%3A07.754-08%3A00; MACHINE_LAST_SEEN=2019-01-20T01%3A45%3A07.754-08%3A00; JSESSIONID=aaa7_J1CBzg-IhiVyNOHw; AMCV_1B6E34B85282A0AC0A490D44%40AdobeOrg=-1303530583%7CMCIDTS%7C17917%7CMCMID%7C42941713042481139142014523461527613808%7CMCAAMLH-1548582313%7C11%7CMCAAMB-1548582313%7CRKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y%7CMCOPTOUT-1547984713s%7CNONE%7CMCAID%7CNONE%7CvVersion%7C3.3.0; s_sq=wileyonlinelibrary-production%3D%2526pid%253DAbout%252520%25257C%252520%2525E8%252589%2525BA%2525E6%25259C%2525AF%2525E4%2525B8%25258E%2525E5%2525BA%252594%2525E7%252594%2525A8%2525E8%252589%2525BA%2525E6%25259C%2525AF%252520%25253Cfont%252520style%25253D%252522color%25253Awhite%25253B%252522%25253E%2528RUC%2529%25253C%25252Ffont%25253E%2526pidt%253D1%2526oid%253Dhttp%25253A%25252F%25252Fonlinelibrary.wiley.com%25252Fbook%25252F10.1002%25252F9780470696118%2526ot%253DA; _gat_wolga=1; JSESSIONID=aaa7_J1CBzg-IhiVyNOHw; _sdsat_MCID=42941713042481139142014523461527613808; __atuvc=2%7C3%2C17%7C4; __atuvs=5c444327a42a2d54002; _ga=GA1.2.1362788368.1547694307; _gid=GA1.2.1790328491.1547943936; s_cc=true");
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
//                //SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "--状态码" + resStatu);
                if (resStatu == HttpStatus.SC_OK) {
                    html = parsingHtmlCaoliu(response);
                    if (html != null) {
                        break;
                    }
                }
            } catch (Exception e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            } finally {
                httpClient.close();
            }
        }
        return html;
    }


    /**
     * 通用的获取html的get方法 无代理
     */
    public static String getHtmlNoProxy(BasicCookieStore cookieStore, HttpClientConfig config) throws IOException {
//        //SmallTool.printMess("当前的地址:" + config.getUrl());
        String html = null;
        for (int i = 1; i <= 3; i++) {
            CloseableHttpClient httpClient = HttpClientBuilder.create()
                    .setDefaultCookieStore(cookieStore)
                    .setDefaultCredentialsProvider(config.getCredsProvider())
                    .build();
            HttpGet httpget = null;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                httpget.addHeader(HttpHeaders.USER_AGENT, config.getUSER_AGENT());
                //添加cookie 测试
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(response);
                    if (html != null) {
                        break;
                    }
                }
            } catch (Exception e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            } finally {
                httpClient.close();
            }
        }
        return html;
    }


    private static void savePicToDisk(InputStream in, String dirPath, String filePath) {
        try {
            File dir = new File(dirPath);
            if (dir == null || !dir.exists()) {
                dir.mkdirs();
            }
            //文件真实路径
            String realPath = dirPath.concat(filePath);
            File file = new File(realPath);
            if (file == null || !file.exists()) {
                file.createNewFile();
            }

            FileOutputStream fos = new FileOutputStream(file);
            byte[] buf = new byte[1024];
            int len = 0;
            while ((len = in.read(buf)) != -1) {
                fos.write(buf, 0, len);
            }
            fos.flush();
            fos.close();

        } catch (IOException e) {
            SmallTool.printMessError("异常信息++" + e.getMessage());
        } finally {
            try {
                in.close();
            } catch (IOException e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            }
        }
    }

    private static boolean savePicToDiskBoolean(InputStream in, String dirPath, String filePath) {
        boolean isRes = true;
        try {
            File dir = new File(dirPath);
            if (dir == null || !dir.exists()) {
                dir.mkdirs();
            }
            //文件真实路径
            String realPath = dirPath.concat(filePath);
            File file = new File(realPath);
            if (file == null || !file.exists()) {
                file.createNewFile();
            }

            FileOutputStream fos = new FileOutputStream(file);
            byte[] buf = new byte[1024];
            int len = 0;
            while ((len = in.read(buf)) != -1) {
                fos.write(buf, 0, len);
            }
            fos.flush();
            fos.close();

        } catch (IOException e) {
            isRes = false;
            SmallTool.printMessError("异常信息++" + e.getMessage());
        } finally {
            try {
                in.close();
            } catch (IOException e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            }
        }
        return isRes;
    }


    // 通过json 发送数据
    public static String postHtmlByJsonStr(BasicCookieStore cookieStore, String json, HttpClientConfig config) throws IOException {
        SmallTool.printMess("httpclient里面的地址" + config.getUrl());
        String html = "";
        for (int i = 1; i <= 3; i++) {
            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
            HttpPost httppost;
            long time1 = System.currentTimeMillis();
            CloseableHttpResponse responce;
            try {
                httppost = new HttpPost(config.getUrl());
                httppost.setEntity(new StringEntity(json, "utf8"));
                httppost.setConfig(config.getRequestConfig());
                httppost.setHeader("Content-type", "application/json; charset=UTF-8");
                httppost.addHeader(HttpHeaders.USER_AGENT, config.getUSER_AGENT());
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httppost.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httppost.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httppost.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                responce = httpClient.execute(httppost);
                int resStatu = responce.getStatusLine().getStatusCode();
                SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "状态码" + resStatu);
                //获得相应实体
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(responce);
                    if (html != null) {
                        break;
                    }
                }
            } catch (Exception e) {
                SmallTool.printMessError("查看异常的字符串" + e.toString());

            } finally {
                httpClient.close();
            }
        }
        return html;
    }

    /**
     * 通用的获取html的post方法
     */
    public static String postHtml(BasicCookieStore cookieStore, Map<String, String> map, HttpClientConfig config) throws IOException {
        SmallTool.printMess("httpclient里面的地址" + config.getUrl());
        String html = "";
        for (int i = 1; i <= 3; i++) {
            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
            HttpPost httppost = null;
            long time1 = System.currentTimeMillis();
            CloseableHttpResponse responce;
            try {
                httppost = new HttpPost(config.getUrl());
                List<NameValuePair> nvps = new ArrayList<NameValuePair>();
                if (map != null) {
                    for (Map.Entry<String, String> entry : map.entrySet()) {
                        nvps.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
                    }
                }
                httppost.setEntity(new UrlEncodedFormEntity(nvps, "utf8"));
                httppost.setConfig(config.getRequestConfig());
                httppost.setHeader("Content-type", "application/x-www-form-urlencoded");
                httppost.addHeader(HttpHeaders.USER_AGENT, config.getUSER_AGENT());
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httppost.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httppost.addHeader(HttpHeaders.HOST, config.getHost());
                }
                responce = httpClient.execute(httppost);
                int resStatu = responce.getStatusLine().getStatusCode();
                SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "状态码" + resStatu);
                //获得相应实体
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(responce);
                    if (html != null) {
                        break;
                    }
                }
            } catch (Exception e) {
                SmallTool.printMessError("查看异常的字符串" + e.toString());
                if (e.toString().contains("java.net.UnknownHostException")) {

                }
                if (e.toString().contains("connect time out")) {
                }
            } finally {
                httpClient.close();
            }
        }
        return html;
    }

    /**
     * 通用的下载图片get方法
     */
    public static void getDownImage(HttpClientConfig config) throws IOException {
        config.setProxy(true);

        BasicCookieStore cookieStore = new BasicCookieStore();
        File file = new File(config.getDirPath() + config.getFilePath());
        if (file.exists()) {
//            SmallTool.printMess("存在不下载 嘻嘻" + config.getFilePath());
        } else {
//            SmallTool.printMess("当前的地址:" + config.getUrl());
            for (int i = 1; i <= 3; i++) {
                CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
                HttpGet httpget = null;
                long time1 = System.currentTimeMillis();
                CloseableHttpResponse response;
                try {
                    httpget = new HttpGet(config.getUrl());
                    httpget.setConfig(config.getRequestConfig());
                    httpget.addHeader(HttpHeaders.USER_AGENT, config.getUSER_AGENT());
                    if (config.getHeaderMap().size() > 0) {
                        Map<String, String> headerMap = config.getHeaderMap();
                        Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                        for (Map.Entry<String, String> entry : entries) {
                            httpget.addHeader(entry.getKey(), entry.getValue());
                        }
                    }
                    response = httpClient.execute(httpget);
                    int resStatu = response.getStatusLine().getStatusCode();
                    if (resStatu == 302) {
                        AVLogWriteUtil.WriteLog(config.getFilePath() + "没有地址是" + config.getUrl(), "");
                        SmallTool.printMess(resStatu);
                        break;
                    }
                    //SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "--状态码" + resStatu);
                    if (resStatu == HttpStatus.SC_OK) {
                        HttpEntity entity = response.getEntity();
                        InputStream in = entity.getContent();
                        savePicToDisk(in, config.getDirPath(), config.getFilePath());
//                        SmallTool.printMess("图片保存成功");
//                        SmallTool.printMess("路径+++" + config.getDirPath() + config.getFilePath());
                        if (in != null) {
                            break;
                        }
                    }
                } finally {
                    httpClient.close();
                }
            }
        }
    }


    public static boolean getDownImageByPmPicture(HttpClientConfig config) throws IOException {
        BasicCookieStore cookieStore = new BasicCookieStore();
        File file = new File(config.getDirPath() + config.getFilePath());
        boolean isHtml = false;
        if (file.exists()) {
            SmallTool.printMess("存在不下载 嘻嘻" + config.getFilePath());
        } else {
//            SmallTool.printMess("当前的地址:" + config.getUrl());
            for (int i = 1; i <= 3; i++) {
                String html;
                CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
                HttpGet httpget = null;
                long time1 = System.currentTimeMillis();
                CloseableHttpResponse response;
                try {
                    httpget = new HttpGet(config.getUrl());
                    httpget.setConfig(config.getRequestConfig());
                    httpget.addHeader(HttpHeaders.USER_AGENT, config.getUSER_AGENT());
                    if (config.getHeaderMap().size() > 0) {
                        Map<String, String> headerMap = config.getHeaderMap();
                        Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                        for (Map.Entry<String, String> entry : entries) {
                            httpget.addHeader(entry.getKey(), entry.getValue());
                        }
                    }
                    response = httpClient.execute(httpget);
                    int resStatu = response.getStatusLine().getStatusCode();
                    if (resStatu == 302) {
                        AVLogWriteUtil.WriteLog(config.getFilePath() + "没有地址是" + config.getUrl(), "");
                        SmallTool.printMess(resStatu);
                        break;
                    }
                    //SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "--状态码" + resStatu);
                    if (resStatu == HttpStatus.SC_OK) {
                        HttpEntity entity = response.getEntity();
                        try {
                            html = HttpClientUtil.parsingHtml(response);
                        } catch (Exception e) {
                            isHtml = true;
                            e.printStackTrace();
                        }
                        if (!isHtml) {
                            InputStream in = entity.getContent();
                            savePicToDisk(in, config.getDirPath(), config.getFilePath());
//                            SmallTool.printMess("图片保存成功");
//                            SmallTool.printMess("路径+++" + config.getDirPath() + config.getFilePath());
                            if (in != null) {
                                break;
                            }
                        }

                    }
                } finally {
                    httpClient.close();
                }
            }
        }
        return isHtml;
    }

    /**
     * 通用的下载图片get方法 返回http 状态码
     */
    public static Integer getDownImageCode(HttpClientConfig config) throws IOException {
        BasicCookieStore cookieStore = new BasicCookieStore();
        Integer code = 404;
        File file = new File(config.getDirPath() + config.getFilePath());
        boolean isExistCorrect = true;
        boolean isExist = true;
        if (file.exists()) {
            //如果文件小于2K 则认为是失败的
            if (FileUtil.size(file) <= 3428L) {
                isExistCorrect = false;
            } else {
                SmallTool.printMess("存在不下载 嘻嘻" + config.getFilePath());
                code = 200;
            }
        } else {
            isExist = false;
        }

        if (!isExist || !isExistCorrect) {
            SmallTool.printMess("当前的地址:" + config.getUrl());
            for (int i = 1; i <= 3; i++) {
                CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
                HttpGet httpget = null;
                long time1 = System.currentTimeMillis();
                CloseableHttpResponse response;
                try {
                    httpget = new HttpGet(config.getUrl());
                    httpget.setConfig(config.getRequestConfig());
                    httpget.addHeader(HttpHeaders.USER_AGENT, config.getUSER_AGENT());
                    if (config.getHeaderMap().size() > 0) {
                        Map<String, String> headerMap = config.getHeaderMap();
                        Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                        for (Map.Entry<String, String> entry : entries) {
                            httpget.addHeader(entry.getKey(), entry.getValue());
                        }
                    }
                    response = httpClient.execute(httpget);
                    int resStatu = response.getStatusLine().getStatusCode();
                    if (resStatu == 302) {
                        AVLogWriteUtil.WriteLog(config.getFilePath() + "没有地址是" + config.getUrl(), "");
                        SmallTool.printMess(resStatu);
                        break;
                    }
                    //SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "--状态码" + resStatu);
                    if (resStatu == HttpStatus.SC_OK) {
                        HttpEntity entity = response.getEntity();
                        InputStream in = entity.getContent();
                        boolean b = savePicToDiskBoolean(in, config.getDirPath(), config.getFilePath());
                        if (b) {
                            //SmallTool.printMess("图片保存成功");
                            SmallTool.printMess("保存成功路径+++" + config.getDirPath() + config.getFilePath());
                            // 进行校验是不是完整的 如果不是则进行提示 查看是不是多线程问题?
                            File fileSave = new File(config.getDirPath() + config.getFilePath());
                            boolean b1 = LocalPmcFileCheck2.checkFileSizeCorrect(fileSave, config.getFileByteSize());
                            if (!b1 && config.getFileByteSize() > 0) {
                                System.out.println("_____________________________________________________________________");
                                SmallTool.printMessError("下载大小不对 请排查+" + config.getDirPath() + config.getFilePath());
                                SmallTool.printMessError("应该大小是+" + config.getFileByteSize() + " 实际大小是" + fileSave.length());
                                System.out.println("+++++++++++++++++++++++++++++++++++++++++++++++++++++++");
                            }

                            code = resStatu;
                        }

                        if (in != null) {
                            break;
                        }
                    } else {
                        SmallTool.printMessError("http error code = " + resStatu);
                        code = resStatu;
                        return code;
                    }
                } catch (Exception e) {
                    SmallTool.printMessError("下载异常++" + config.getUrl());
                } finally {
                    httpClient.close();
                }
            }
        }
        return code;
    }


    /**
     * 通用的下载图片get方法
     */
    public static void getImage(BasicCookieStore cookieStore, HttpClientConfig config, String dirPath, String filePath) throws IOException {
        File file = new File(dirPath + filePath);
        if (file.exists()) {
//            SmallTool.printMess("存在不下载 嘻嘻" + filePath);
        } else {
            //SmallTool.printMess("当前的地址:" + config.getUrl());
            for (int i = 1; i <= 6; i++) {
                HttpHost httpHost = new HttpHost(config.getProxyIp(), config.getProxyPort());
                CloseableHttpClient httpClient = HttpClientBuilder.create()
                        .setProxy(httpHost)
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider()).build();
                HttpGet httpget = null;
                long time1 = System.currentTimeMillis();
                CloseableHttpResponse response;
                try {
                    httpget = new HttpGet(config.getUrl());
                    httpget.setConfig(config.getRequestConfig());
                    httpget.addHeader(HttpHeaders.USER_AGENT, config.getUSER_AGENT());
                    if (config.getHeaderMap().size() > 0) {
                        Map<String, String> headerMap = config.getHeaderMap();
                        Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                        for (Map.Entry<String, String> entry : entries) {
                            httpget.addHeader(entry.getKey(), entry.getValue());
                        }
                    }
                    response = httpClient.execute(httpget);
                    int resStatu = response.getStatusLine().getStatusCode();
                    if (resStatu == 302) {
                        AVLogWriteUtil.WriteLog(filePath + "没有地址是" + config.getUrl(), "");
                        SmallTool.printMess(resStatu);
                        break;
                    }
                    //SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "--状态码" + resStatu);
                    if (resStatu == HttpStatus.SC_OK) {
                        HttpEntity entity = response.getEntity();
                        InputStream in = entity.getContent();
                        savePicToDisk(in, dirPath, filePath);
                        //SmallTool.printMess("图片保存成功");
//                        SmallTool.printMess("路径+++" + dirPath + filePath);
                        if (in != null) {
                            break;
                        }
                    }
                } catch (Exception e) {
                    SmallTool.printMessError("下载异常++" + config.getUrl());

                } finally {
                    httpClient.close();
                }
            }
        }
    }


    public static CloseableHttpClient createSSLClientDefault(BasicCookieStore basicCookieStore, HttpClientConfig clientConfig) {
        HttpHost httpHost = null;
        CloseableHttpClient httpClient;
        if (clientConfig.isProxy()) {
            httpHost = new HttpHost(clientConfig.getProxyIp(), clientConfig.getProxyPort());
        }
        if (clientConfig.getSslUse()) {
            try {
                //使用 loadTrustMaterial() 方法实现一个信任策略，信任所有证书
                // 信任所有
                SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true).build();
                //NoopHostnameVerifier类:  作为主机名验证工具，实质上关闭了主机名验证，它接受任何
                //有效的SSL会话并匹配到目标主机。
                HostnameVerifier hostnameVerifier = NoopHostnameVerifier.INSTANCE;
                SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
                if (clientConfig.isProxy()) {
                    SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(clientConfig.getTIME_OUT()).build();
                    CloseableHttpClient build = HttpClients.custom()
                            .setProxy(httpHost)
                            .setDefaultCookieStore(basicCookieStore)
                            .setDefaultSocketConfig(socketConfig)
                            .setSSLSocketFactory(sslsf).build();
                    return build;
                } else {
                    CloseableHttpClient build = HttpClients.custom().setUserAgent(clientConfig.getUSER_AGENT()).setDefaultCookieStore(basicCookieStore).setSSLSocketFactory(sslsf).build();
                    return build;
                }
            } catch (KeyManagementException e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            } catch (NoSuchAlgorithmException e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            } catch (KeyStoreException e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            }
        } else {
            if (UrlEntity.DailiFalse) {
                SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(clientConfig.getTIME_OUT()).build();
                httpClient = HttpClientBuilder.create()
                        .setProxy(httpHost)
                        .setDefaultSocketConfig(socketConfig)
                        .setDefaultCookieStore(clientConfig.getCookieStore())
                        .setDefaultCredentialsProvider(clientConfig.getCredsProvider())
                        .build();
                return httpClient;
            } else {
                httpClient = HttpClientBuilder.create()
                        .setDefaultCookieStore(clientConfig.getCookieStore())
                        .setDefaultCredentialsProvider(clientConfig.getCredsProvider())
                        .build();
                return httpClient;
            }

        }
        return HttpClients.createDefault();
    }


    /**
     * 通用的获取html的get方法
     *
     * @param url
     * @param param 用于草榴下载编码用
     * @return
     * @throws IOException
     */
    public static String getHtmlDirect(String url, String param) throws IOException {
        BasicCookieStore cookieStore = new BasicCookieStore();
        HttpClientConfig config = HttpClientUtil.getSearConfig(url, "", "", true);
        String html = null;
        if (!UrlEntity.DailiFalse) {
            config.setProxy(false);
        } else {
            config.setProxy(true);
        }


        CloseableHttpClient httpClient;

        for (int i = 1; i <= 3; i++) {
            if (config.isProxy()) {
                HttpHost httpHost = new HttpHost(config.getProxyIp(), config.getProxyPort());

                httpClient = HttpClientBuilder.create()
                        .setProxy(httpHost)
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();

            } else {
                httpClient = HttpClientBuilder.create()
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();
            }
//            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                HttpClientConfig clientConfig = new HttpClientConfig();
                httpget.addHeader(HttpHeaders.USER_AGENT, clientConfig.getUSER_AGENT());
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    if (StringUtils.isNotEmpty(param) && param.equals("caoliu")) {
                        html = parsingHtmlCaoliu(response);
                    } else {
                        html = HttpClientUtil.parsingHtml(response);

                    }
                    if (html != null) {
                        break;
                    }
                }
            } catch (Exception e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            } finally {
                httpClient.close();
            }
        }
        return html;
    }


    /**
     * 通用的获取html的get方法
     */
    public static String getHtmlDirectProxy(String url) throws IOException {
        BasicCookieStore cookieStore = new BasicCookieStore();
        HttpClientConfig config = HttpClientUtil.getSearConfig(url, "", "", true);
        config.setProxy(true);
        config.setSslUse(true);
        String html = null;
        CloseableHttpClient httpClient;

        for (int i = 1; i <= 3; i++) {

            if (config.isProxy()) {
                HttpHost httpHost = new HttpHost(config.getProxyIp(), config.getProxyPort());

                httpClient = HttpClientBuilder.create()
                        .setProxy(httpHost)
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();

            } else {
                httpClient = HttpClientBuilder.create()
                        .setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(config.getCredsProvider())
                        .build();
            }
//            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);

            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                HttpClientConfig clientConfig = new HttpClientConfig();

                httpget.addHeader(HttpHeaders.USER_AGENT, clientConfig.getUSER_AGENT());
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(response);
                    if (html != null) {
                        break;
                    }
                }
            } catch (Exception e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            } finally {
                httpClient.close();
            }
        }
        return html;
    }


    /**
     * 通用的获取url 302 跳转地址 的get方法
     */
    public static String getDirect302Url(String url) throws IOException {
        BasicCookieStore cookieStore = new BasicCookieStore();
        HttpClientConfig config = HttpClientUtil.getSearConfig(url, "", "", false);
        config.setProxy(false);
        String html = "";
        CloseableHttpClient httpClient;
        T:
        for (int i = 1; i <= 1; i++) {
            httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                HttpClientConfig clientConfig = new HttpClientConfig();

                httpget.addHeader(HttpHeaders.USER_AGENT, clientConfig.getUSER_AGENT());
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_MOVED_PERMANENTLY || resStatu == HttpStatus.SC_MOVED_TEMPORARILY) {
                    Header[] allHeaders = response.getAllHeaders();
                    for (Header allHeader : allHeaders) {
                        String name = allHeader.getName();
                        if (name.toLowerCase().contains("location")) {
                            String value = allHeader.getValue();
                            if (StringUtils.isNotEmpty(value)) {
                                html = value;
                                break T;
                            }

                        }
                    }
                } else if (resStatu == HttpStatus.SC_NOT_FOUND) {
                    html = "404";
                    break T;
                }
            } catch (Exception e) {
//                SmallTool.printMessError("异常信息++" + e.getMessage());
            } finally {
                httpClient.close();
            }
        }
        return html;
    }

    /**
     * 通用的获取url 302 跳转地址 的post方法
     */
    public static String postDirect302Url(String url, Map<String, String> map) throws IOException {
        // https://dx.doi.org 此post 专用   referer: https://dx.doi.org/ origin: https://dx.doi.org
        BasicCookieStore cookieStore = new BasicCookieStore();
        HttpClientConfig config = HttpClientUtil.getSearConfig(url, "https://dx.doi.org/", "dx.doi.org", false);
        config.setProxy(true);
        config.setSslUse(true);
        String html = "";
        CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
        T:
        for (int i = 1; i <= 1; i++) {
            HttpPost httppost;
            CloseableHttpResponse response;
            try {
                httppost = new HttpPost(config.getUrl());
                List<NameValuePair> nvps = new ArrayList<>();
                if (map != null) {
                    for (Map.Entry<String, String> entry : map.entrySet()) {
                        nvps.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
                    }
                }
                httppost.setEntity(new UrlEncodedFormEntity(nvps, "utf8"));
                httppost.setConfig(config.getRequestConfig());
                httppost.setHeader("Content-type", "application/x-www-form-urlencoded");
                HttpClientConfig clientConfig = new HttpClientConfig();

                httppost.addHeader(HttpHeaders.USER_AGENT, clientConfig.getUSER_AGENT());
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httppost.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httppost.addHeader(HttpHeaders.HOST, config.getHost());
                }
                response = httpClient.execute(httppost);
                int resStatu = response.getStatusLine().getStatusCode();
//                SmallTool.printMess("httpclient耗时" + (System.currentTimeMillis() - time1) + "状态码" + resStatu);
                if (resStatu == HttpStatus.SC_MOVED_PERMANENTLY || resStatu == HttpStatus.SC_MOVED_TEMPORARILY) {
                    Header[] allHeaders = response.getAllHeaders();
                    for (Header allHeader : allHeaders) {
                        String name = allHeader.getName();
                        if (name.toLowerCase().contains("location")) {
                            String value = allHeader.getValue();
                            if (StringUtils.isNotEmpty(value)) {
                                html = value;
                                break T;
                            }

                        }
                    }
                } else if (resStatu == HttpStatus.SC_NOT_FOUND) {
                    html = "404";
                    break T;
                }
            } catch (Exception e) {
//                SmallTool.printMessError("查看异常的字符串" + e.toString());
                if (e.toString().contains("java.net.UnknownHostException")) {

                }
                if (e.toString().contains("connect time out")) {
                }
            } finally {
                httpClient.close();
            }
        }
        return html;
    }

    /**
     * 通用的获取html的get方法
     */
    public static String getHtmlDirect(String url) throws IOException {
        BasicCookieStore cookieStore = new BasicCookieStore();
        HttpClientConfig config = HttpClientUtil.getSearConfig(url, "", "", true);
        config.setProxy(false);
        String html = null;
        CloseableHttpClient httpClient;

        for (int i = 1; i <= 2; i++) {

//            if (config.isProxy()) {
//                HttpHost httpHost = new HttpHost(config.getProxyIp(), config.getProxyPort());
//
//                httpClient = HttpClientBuilder.create()
//                        .setProxy(httpHost)
//                        .setDefaultCookieStore(cookieStore)
//                        .setDefaultCredentialsProvider(config.getCredsProvider())
//                        .build();
//
//            } else {
//                httpClient = HttpClientBuilder.create()
//                        .setDefaultCookieStore(cookieStore)
//                        .setDefaultCredentialsProvider(config.getCredsProvider())
//                        .build();
//            }
            httpClient = HttpClient.createSSLClientDefault(cookieStore, config);

            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                HttpClientConfig clientConfig = new HttpClientConfig();
                httpget.addHeader(HttpHeaders.USER_AGENT, clientConfig.getUSER_AGENT());
//                httpget.addHeader("sec-ch-ua", "\"Chromium\";v=\"94\", \"Microsoft Edge\";v=\"94\", \";Not A Brand\";v=\"99\"");
//                httpget.addHeader("sec-ch-ua-platform", "\"Windows\"");
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(response);
                    if (html != null) {
                        break;
                    }
                } else {
                    System.out.println(resStatu);
                }
            } catch (Exception e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            } finally {
                httpClient.close();
            }
        }
        return html;
    }

    /**
     * 通用的获取html的get方法
     */
    public static String getHtml(BasicCookieStore cookieStore, HttpClientConfig config) throws IOException {
        String html = null;
        for (int i = 1; i <= 3; i++) {
            CloseableHttpClient httpClient = HttpClient.createSSLClientDefault(cookieStore, config);
            HttpGet httpget;
            CloseableHttpResponse response;
            try {
                httpget = new HttpGet(config.getUrl());
                httpget.setConfig(config.getRequestConfig());
                httpget.addHeader(HttpHeaders.USER_AGENT, config.getUSER_AGENT());
                if (!StringUtils.isEmpty(config.getRefer())) {
                    httpget.addHeader(HttpHeaders.REFERER, config.getRefer());
                }
                if (!StringUtils.isEmpty(config.getHost())) {
                    httpget.addHeader(HttpHeaders.HOST, config.getHost());
                }
                if (config.getHeaderMap().size() > 0) {
                    Map<String, String> headerMap = config.getHeaderMap();
                    Set<Map.Entry<String, String>> entries = headerMap.entrySet();
                    for (Map.Entry<String, String> entry : entries) {
                        httpget.addHeader(entry.getKey(), entry.getValue());
                    }
                }
                response = httpClient.execute(httpget);
                int resStatu = response.getStatusLine().getStatusCode();
                if (resStatu == HttpStatus.SC_OK) {
                    html = HttpClientUtil.parsingHtml(response);
                    if (html != null) {
                        break;
                    }
                }
            } catch (Exception e) {
                SmallTool.printMessError("异常信息++" + e.getMessage());
            } finally {
                httpClient.close();
            }
        }
        return html;
    }


}
